{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['cifar-10-python.tar.gz']\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "\n",
    "import os\n",
    "print(os.listdir(\"../input\"))\n",
    "\n",
    "import time\n",
    "\n",
    "# import pytorch\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "from torch.optim import SGD,Adam,lr_scheduler\n",
    "from torch.utils.data import random_split\n",
    "import torchvision\n",
    "from torchvision import transforms, datasets\n",
    "from torch.utils.data import DataLoader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# define transformations for train\n",
    "train_transform = transforms.Compose([\n",
    "    transforms.RandomHorizontalFlip(p=.40),\n",
    "    transforms.RandomRotation(30),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define transformations for test\n",
    "test_transform = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize([0.5, 0.5, 0.5], [0.5, 0.5, 0.5])])\n",
    "\n",
    "# define training dataloader\n",
    "def get_training_dataloader(train_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        train_transform: transfroms for train dataset\n",
    "        path: path to cifar100 training python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: train_data_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_train = train_transform\n",
    "    cifar10_training = torchvision.datasets.CIFAR10(root='.', train=True, download=True, transform=transform_train)\n",
    "    cifar10_training_loader = DataLoader(\n",
    "        cifar10_training, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_training_loader\n",
    "\n",
    "# define test dataloader\n",
    "def get_testing_dataloader(test_transform, batch_size=128, num_workers=0, shuffle=True):\n",
    "    \"\"\" return training dataloader\n",
    "    Args:\n",
    "        test_transform: transforms for test dataset\n",
    "        path: path to cifar100 test python dataset\n",
    "        batch_size: dataloader batchsize\n",
    "        num_workers: dataloader num_works\n",
    "        shuffle: whether to shuffle \n",
    "    Returns: cifar100_test_loader:torch dataloader object\n",
    "    \"\"\"\n",
    "\n",
    "    transform_test = test_transform\n",
    "    cifar10_test = torchvision.datasets.CIFAR10(root='.', train=False, download=True, transform=transform_test)\n",
    "    cifar10_test_loader = DataLoader(\n",
    "        cifar10_test, shuffle=shuffle, num_workers=num_workers, batch_size=batch_size)\n",
    "\n",
    "    return cifar10_test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement mish activation function\n",
    "def f_mish(input):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "    '''\n",
    "    return input * torch.tanh(F.softplus(input))\n",
    "\n",
    "# implement class wrapper for mish activation function\n",
    "class mish(nn.Module):\n",
    "    '''\n",
    "    Applies the mish function element-wise:\n",
    "    mish(x) = x * tanh(softplus(x)) = x * tanh(ln(1 + exp(x)))\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = mish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_mish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# implement swish activation function\n",
    "def f_swish(input):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "    '''\n",
    "    return input * torch.sigmoid(input)\n",
    "\n",
    "# implement class wrapper for swish activation function\n",
    "class swish(nn.Module):\n",
    "    '''\n",
    "    Applies the swish function element-wise:\n",
    "    swish(x) = x * sigmoid(x)\n",
    "\n",
    "    Shape:\n",
    "        - Input: (N, *) where * means, any number of additional\n",
    "          dimensions\n",
    "        - Output: (N, *), same shape as the input\n",
    "\n",
    "    Examples:\n",
    "        >>> m = swish()\n",
    "        >>> input = torch.randn(2)\n",
    "        >>> output = m(input)\n",
    "\n",
    "    '''\n",
    "    def __init__(self):\n",
    "        '''\n",
    "        Init method.\n",
    "        '''\n",
    "        super().__init__()\n",
    "\n",
    "    def forward(self, input):\n",
    "        '''\n",
    "        Forward pass of the function.\n",
    "        '''\n",
    "        return f_swish(input)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [],
   "source": [
    "#\"\"\"Bottleneck layers. Although each layer only produces k\n",
    "#output feature-maps, it typically has many more inputs. It\n",
    "#has been noted in [37, 11] that a 1×1 convolution can be in-\n",
    "#troduced as bottleneck layer before each 3×3 convolution\n",
    "#to reduce the number of input feature-maps, and thus to\n",
    "#improve computational efficiency.\"\"\"\n",
    "class Bottleneck(nn.Module):\n",
    "    def __init__(self, in_channels, growth_rate, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        #\"\"\"In  our experiments, we let each 1×1 convolution \n",
    "        #produce 4k feature-maps.\"\"\"\n",
    "        inner_channel = 4 * growth_rate\n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        #\"\"\"We find this design especially effective for DenseNet and \n",
    "        #we refer to our network with such a bottleneck layer, i.e., \n",
    "        #to the BN-ReLU-Conv(1×1)-BN-ReLU-Conv(3×3) version of H ` , \n",
    "        #as DenseNet-B.\"\"\"\n",
    "        self.bottle_neck = nn.Sequential(\n",
    "            nn.BatchNorm2d(in_channels),\n",
    "            f_activation,\n",
    "            nn.Conv2d(in_channels, inner_channel, kernel_size=1, bias=False),\n",
    "            nn.BatchNorm2d(inner_channel),\n",
    "            nn.ReLU(inplace=True),\n",
    "            nn.Conv2d(inner_channel, growth_rate, kernel_size=3, padding=1, bias=False)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return torch.cat([x, self.bottle_neck(x)], 1)\n",
    "\n",
    "#\"\"\"We refer to layers between blocks as transition\n",
    "#layers, which do convolution and pooling.\"\"\"\n",
    "class Transition(nn.Module):\n",
    "    def __init__(self, in_channels, out_channels):\n",
    "        super().__init__()\n",
    "        #\"\"\"The transition layers used in our experiments \n",
    "        #consist of a batch normalization layer and an 1×1 \n",
    "        #convolutional layer followed by a 2×2 average pooling \n",
    "        #layer\"\"\".\n",
    "        self.down_sample = nn.Sequential(\n",
    "            nn.BatchNorm2d(in_channels),\n",
    "            nn.Conv2d(in_channels, out_channels, 1, bias=False),\n",
    "            nn.AvgPool2d(2, stride=2)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        return self.down_sample(x)\n",
    "\n",
    "#DesneNet-BC\n",
    "#B stands for bottleneck layer(BN-RELU-CONV(1x1)-BN-RELU-CONV(3x3))\n",
    "#C stands for compression factor(0<=theta<=1)\n",
    "class DenseNet(nn.Module):\n",
    "    def __init__(self, block, nblocks, growth_rate=12, reduction=0.5, num_class=100, activation = 'relu'):\n",
    "        super().__init__()\n",
    "        self.growth_rate = growth_rate\n",
    "\n",
    "        #\"\"\"Before entering the first dense block, a convolution \n",
    "        #with 16 (or twice the growth rate for DenseNet-BC) \n",
    "        #output channels is performed on the input images.\"\"\"\n",
    "        inner_channels = 2 * growth_rate\n",
    "\n",
    "        #For convolutional layers with kernel size 3×3, each \n",
    "        #side of the inputs is zero-padded by one pixel to keep \n",
    "        #the feature-map size fixed.\n",
    "        self.conv1 = nn.Conv2d(3, inner_channels, kernel_size=3, padding=1, bias=False) \n",
    "        \n",
    "        if activation == 'relu':\n",
    "            f_activation = nn.ReLU(inplace=True)\n",
    "            \n",
    "        if activation == 'swish':\n",
    "            f_activation = swish()\n",
    "            \n",
    "        if activation == 'mish':\n",
    "            f_activation = mish()\n",
    "\n",
    "        self.features = nn.Sequential()\n",
    "\n",
    "        for index in range(len(nblocks) - 1):\n",
    "            self.features.add_module(\"dense_block_layer_{}\".format(index), self._make_dense_layers(block, inner_channels, nblocks[index]))\n",
    "            inner_channels += growth_rate * nblocks[index]\n",
    "\n",
    "            #\"\"\"If a dense block contains m feature-maps, we let the \n",
    "            #following transition layer generate θm output feature-\n",
    "            #maps, where 0 < θ ≤ 1 is referred to as the compression \n",
    "            #fac-tor.\n",
    "            out_channels = int(reduction * inner_channels) # int() will automatic floor the value\n",
    "            self.features.add_module(\"transition_layer_{}\".format(index), Transition(inner_channels, out_channels))\n",
    "            inner_channels = out_channels\n",
    "\n",
    "        self.features.add_module(\"dense_block{}\".format(len(nblocks) - 1), self._make_dense_layers(block, inner_channels, nblocks[len(nblocks)-1]))\n",
    "        inner_channels += growth_rate * nblocks[len(nblocks) - 1]\n",
    "        self.features.add_module('bn', nn.BatchNorm2d(inner_channels))\n",
    "        self.features.add_module('activation', f_activation)\n",
    "\n",
    "        self.avgpool = nn.AdaptiveAvgPool2d((1, 1))\n",
    "\n",
    "        self.linear = nn.Linear(inner_channels, num_class)\n",
    "\n",
    "    def forward(self, x):\n",
    "        output = self.conv1(x)\n",
    "        output = self.features(output)\n",
    "        output = self.avgpool(output)\n",
    "        output = output.view(output.size()[0], -1)\n",
    "        output = self.linear(output)\n",
    "        return output\n",
    "\n",
    "    def _make_dense_layers(self, block, in_channels, nblocks):\n",
    "        dense_block = nn.Sequential()\n",
    "        for index in range(nblocks):\n",
    "            dense_block.add_module('bottle_neck_layer_{}'.format(index), block(in_channels, self.growth_rate))\n",
    "            in_channels += self.growth_rate\n",
    "        return dense_block\n",
    "\n",
    "def densenet121(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,24,16], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet169(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,32,32], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet201(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,48,32], growth_rate=32, activation = activation)\n",
    "\n",
    "def densenet161(activation = 'relu'):\n",
    "    return DenseNet(Bottleneck, [6,12,36,24], growth_rate=48, activation = activation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading https://www.cs.toronto.edu/~kriz/cifar-10-python.tar.gz to ./cifar-10-python.tar.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "170500096it [00:06, 27266762.88it/s]                               \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "trainloader = get_training_dataloader(train_transform)\n",
    "testloader = get_testing_dataloader(test_transform)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "device(type='cuda', index=0)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "epochs = 100\n",
    "batch_size = 128\n",
    "learning_rate = 0.001\n",
    "device = torch.device('cuda:0' if torch.cuda.is_available() else \"cpu\")\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = densenet121(activation = 'mish')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# set loss function\n",
    "criterion = nn.CrossEntropyLoss()\n",
    "\n",
    "# set optimizer, only train the classifier parameters, feature parameters are frozen\n",
    "optimizer = Adam(model.parameters(), lr=learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats = pd.DataFrame(columns = ['Epoch', 'Time per epoch', 'Avg time per step', 'Train loss', 'Train accuracy', 'Train top-3 accuracy','Test loss', 'Test accuracy', 'Test top-3 accuracy']) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch 1/100.. Time per epoch: 114.7561.. Average time per step: 0.2935.. Train loss: 1.5249.. Train accuracy: 0.4597.. Top-3 train accuracy: 0.7894.. Test loss: 1.2601.. Test accuracy: 0.5349.. Top-3 test accuracy: 0.8660\n",
      "Epoch 2/100.. Time per epoch: 113.8710.. Average time per step: 0.2912.. Train loss: 1.0853.. Train accuracy: 0.6126.. Top-3 train accuracy: 0.8841.. Test loss: 1.0490.. Test accuracy: 0.6337.. Top-3 test accuracy: 0.8893\n",
      "Epoch 3/100.. Time per epoch: 113.6356.. Average time per step: 0.2906.. Train loss: 0.9157.. Train accuracy: 0.6728.. Top-3 train accuracy: 0.9132.. Test loss: 0.8429.. Test accuracy: 0.7047.. Top-3 test accuracy: 0.9303\n",
      "Epoch 4/100.. Time per epoch: 113.4762.. Average time per step: 0.2902.. Train loss: 0.7827.. Train accuracy: 0.7243.. Top-3 train accuracy: 0.9327.. Test loss: 0.7109.. Test accuracy: 0.7474.. Top-3 test accuracy: 0.9469\n",
      "Epoch 5/100.. Time per epoch: 113.1320.. Average time per step: 0.2893.. Train loss: 0.6935.. Train accuracy: 0.7551.. Top-3 train accuracy: 0.9450.. Test loss: 0.5803.. Test accuracy: 0.7950.. Top-3 test accuracy: 0.9626\n",
      "Epoch 6/100.. Time per epoch: 113.3381.. Average time per step: 0.2899.. Train loss: 0.6190.. Train accuracy: 0.7829.. Top-3 train accuracy: 0.9539.. Test loss: 0.5830.. Test accuracy: 0.7931.. Top-3 test accuracy: 0.9609\n",
      "Epoch 7/100.. Time per epoch: 112.9349.. Average time per step: 0.2888.. Train loss: 0.5569.. Train accuracy: 0.8050.. Top-3 train accuracy: 0.9628.. Test loss: 0.5315.. Test accuracy: 0.8181.. Top-3 test accuracy: 0.9656\n",
      "Epoch 8/100.. Time per epoch: 113.1013.. Average time per step: 0.2893.. Train loss: 0.5201.. Train accuracy: 0.8163.. Top-3 train accuracy: 0.9665.. Test loss: 0.4765.. Test accuracy: 0.8342.. Top-3 test accuracy: 0.9714\n",
      "Epoch 9/100.. Time per epoch: 113.2049.. Average time per step: 0.2895.. Train loss: 0.4713.. Train accuracy: 0.8343.. Top-3 train accuracy: 0.9711.. Test loss: 0.4833.. Test accuracy: 0.8355.. Top-3 test accuracy: 0.9715\n",
      "Epoch 10/100.. Time per epoch: 113.1739.. Average time per step: 0.2894.. Train loss: 0.4404.. Train accuracy: 0.8446.. Top-3 train accuracy: 0.9752.. Test loss: 0.4747.. Test accuracy: 0.8379.. Top-3 test accuracy: 0.9723\n",
      "Epoch 11/100.. Time per epoch: 113.0693.. Average time per step: 0.2892.. Train loss: 0.4104.. Train accuracy: 0.8561.. Top-3 train accuracy: 0.9771.. Test loss: 0.4230.. Test accuracy: 0.8554.. Top-3 test accuracy: 0.9783\n",
      "Epoch 12/100.. Time per epoch: 112.9608.. Average time per step: 0.2889.. Train loss: 0.3875.. Train accuracy: 0.8629.. Top-3 train accuracy: 0.9784.. Test loss: 0.3913.. Test accuracy: 0.8660.. Top-3 test accuracy: 0.9782\n",
      "Epoch 13/100.. Time per epoch: 113.2515.. Average time per step: 0.2896.. Train loss: 0.3582.. Train accuracy: 0.8752.. Top-3 train accuracy: 0.9813.. Test loss: 0.4265.. Test accuracy: 0.8568.. Top-3 test accuracy: 0.9760\n",
      "Epoch 14/100.. Time per epoch: 113.2924.. Average time per step: 0.2898.. Train loss: 0.3347.. Train accuracy: 0.8825.. Top-3 train accuracy: 0.9834.. Test loss: 0.4067.. Test accuracy: 0.8620.. Top-3 test accuracy: 0.9781\n",
      "Epoch 15/100.. Time per epoch: 113.3064.. Average time per step: 0.2898.. Train loss: 0.3137.. Train accuracy: 0.8894.. Top-3 train accuracy: 0.9860.. Test loss: 0.3585.. Test accuracy: 0.8799.. Top-3 test accuracy: 0.9814\n",
      "Epoch 16/100.. Time per epoch: 113.1485.. Average time per step: 0.2894.. Train loss: 0.2996.. Train accuracy: 0.8941.. Top-3 train accuracy: 0.9865.. Test loss: 0.3494.. Test accuracy: 0.8819.. Top-3 test accuracy: 0.9828\n",
      "Epoch 17/100.. Time per epoch: 112.9719.. Average time per step: 0.2889.. Train loss: 0.2832.. Train accuracy: 0.8982.. Top-3 train accuracy: 0.9893.. Test loss: 0.3523.. Test accuracy: 0.8823.. Top-3 test accuracy: 0.9831\n",
      "Epoch 18/100.. Time per epoch: 112.8840.. Average time per step: 0.2887.. Train loss: 0.2598.. Train accuracy: 0.9083.. Top-3 train accuracy: 0.9900.. Test loss: 0.3484.. Test accuracy: 0.8841.. Top-3 test accuracy: 0.9831\n",
      "Epoch 19/100.. Time per epoch: 113.0763.. Average time per step: 0.2892.. Train loss: 0.2405.. Train accuracy: 0.9154.. Top-3 train accuracy: 0.9914.. Test loss: 0.3567.. Test accuracy: 0.8864.. Top-3 test accuracy: 0.9809\n",
      "Epoch 20/100.. Time per epoch: 113.2359.. Average time per step: 0.2896.. Train loss: 0.2316.. Train accuracy: 0.9174.. Top-3 train accuracy: 0.9914.. Test loss: 0.3547.. Test accuracy: 0.8831.. Top-3 test accuracy: 0.9842\n",
      "Epoch 21/100.. Time per epoch: 112.9907.. Average time per step: 0.2890.. Train loss: 0.2177.. Train accuracy: 0.9217.. Top-3 train accuracy: 0.9929.. Test loss: 0.3709.. Test accuracy: 0.8855.. Top-3 test accuracy: 0.9845\n",
      "Epoch 22/100.. Time per epoch: 113.1076.. Average time per step: 0.2893.. Train loss: 0.2038.. Train accuracy: 0.9269.. Top-3 train accuracy: 0.9936.. Test loss: 0.3431.. Test accuracy: 0.8921.. Top-3 test accuracy: 0.9848\n",
      "Epoch 23/100.. Time per epoch: 113.0958.. Average time per step: 0.2892.. Train loss: 0.1917.. Train accuracy: 0.9318.. Top-3 train accuracy: 0.9944.. Test loss: 0.3511.. Test accuracy: 0.8875.. Top-3 test accuracy: 0.9859\n",
      "Epoch 24/100.. Time per epoch: 112.8873.. Average time per step: 0.2887.. Train loss: 0.1842.. Train accuracy: 0.9333.. Top-3 train accuracy: 0.9949.. Test loss: 0.3731.. Test accuracy: 0.8878.. Top-3 test accuracy: 0.9845\n",
      "Epoch 25/100.. Time per epoch: 113.1641.. Average time per step: 0.2894.. Train loss: 0.1765.. Train accuracy: 0.9371.. Top-3 train accuracy: 0.9951.. Test loss: 0.3640.. Test accuracy: 0.8898.. Top-3 test accuracy: 0.9854\n",
      "Epoch 26/100.. Time per epoch: 112.8886.. Average time per step: 0.2887.. Train loss: 0.1636.. Train accuracy: 0.9425.. Top-3 train accuracy: 0.9957.. Test loss: 0.3630.. Test accuracy: 0.8918.. Top-3 test accuracy: 0.9838\n",
      "Epoch 27/100.. Time per epoch: 112.8690.. Average time per step: 0.2887.. Train loss: 0.1568.. Train accuracy: 0.9427.. Top-3 train accuracy: 0.9965.. Test loss: 0.3290.. Test accuracy: 0.9024.. Top-3 test accuracy: 0.9866\n",
      "Epoch 28/100.. Time per epoch: 112.8992.. Average time per step: 0.2887.. Train loss: 0.1453.. Train accuracy: 0.9476.. Top-3 train accuracy: 0.9968.. Test loss: 0.3274.. Test accuracy: 0.9045.. Top-3 test accuracy: 0.9871\n",
      "Epoch 29/100.. Time per epoch: 112.9478.. Average time per step: 0.2889.. Train loss: 0.1407.. Train accuracy: 0.9505.. Top-3 train accuracy: 0.9967.. Test loss: 0.3409.. Test accuracy: 0.8983.. Top-3 test accuracy: 0.9871\n",
      "Epoch 30/100.. Time per epoch: 112.8714.. Average time per step: 0.2887.. Train loss: 0.1304.. Train accuracy: 0.9533.. Top-3 train accuracy: 0.9970.. Test loss: 0.3699.. Test accuracy: 0.8954.. Top-3 test accuracy: 0.9849\n",
      "Epoch 31/100.. Time per epoch: 112.9509.. Average time per step: 0.2889.. Train loss: 0.1249.. Train accuracy: 0.9556.. Top-3 train accuracy: 0.9975.. Test loss: 0.3404.. Test accuracy: 0.9046.. Top-3 test accuracy: 0.9861\n",
      "Epoch 32/100.. Time per epoch: 113.2569.. Average time per step: 0.2897.. Train loss: 0.1161.. Train accuracy: 0.9592.. Top-3 train accuracy: 0.9978.. Test loss: 0.3706.. Test accuracy: 0.8959.. Top-3 test accuracy: 0.9860\n",
      "Epoch 33/100.. Time per epoch: 113.2780.. Average time per step: 0.2897.. Train loss: 0.1093.. Train accuracy: 0.9610.. Top-3 train accuracy: 0.9982.. Test loss: 0.3654.. Test accuracy: 0.9023.. Top-3 test accuracy: 0.9861\n",
      "Epoch 34/100.. Time per epoch: 112.9745.. Average time per step: 0.2889.. Train loss: 0.1119.. Train accuracy: 0.9605.. Top-3 train accuracy: 0.9980.. Test loss: 0.3609.. Test accuracy: 0.9054.. Top-3 test accuracy: 0.9867\n",
      "Epoch 35/100.. Time per epoch: 112.8893.. Average time per step: 0.2887.. Train loss: 0.1029.. Train accuracy: 0.9638.. Top-3 train accuracy: 0.9983.. Test loss: 0.3612.. Test accuracy: 0.9032.. Top-3 test accuracy: 0.9859\n",
      "Epoch 36/100.. Time per epoch: 112.7163.. Average time per step: 0.2883.. Train loss: 0.0998.. Train accuracy: 0.9637.. Top-3 train accuracy: 0.9980.. Test loss: 0.3929.. Test accuracy: 0.9006.. Top-3 test accuracy: 0.9860\n",
      "Epoch 37/100.. Time per epoch: 112.9249.. Average time per step: 0.2888.. Train loss: 0.0946.. Train accuracy: 0.9662.. Top-3 train accuracy: 0.9987.. Test loss: 0.3710.. Test accuracy: 0.9032.. Top-3 test accuracy: 0.9847\n",
      "Epoch 38/100.. Time per epoch: 112.7779.. Average time per step: 0.2884.. Train loss: 0.0911.. Train accuracy: 0.9669.. Top-3 train accuracy: 0.9986.. Test loss: 0.3818.. Test accuracy: 0.9007.. Top-3 test accuracy: 0.9863\n",
      "Epoch 39/100.. Time per epoch: 112.7109.. Average time per step: 0.2883.. Train loss: 0.0911.. Train accuracy: 0.9673.. Top-3 train accuracy: 0.9987.. Test loss: 0.3730.. Test accuracy: 0.9053.. Top-3 test accuracy: 0.9865\n",
      "Epoch 40/100.. Time per epoch: 112.8520.. Average time per step: 0.2886.. Train loss: 0.0865.. Train accuracy: 0.9688.. Top-3 train accuracy: 0.9990.. Test loss: 0.3838.. Test accuracy: 0.9006.. Top-3 test accuracy: 0.9874\n",
      "Epoch 41/100.. Time per epoch: 113.1068.. Average time per step: 0.2893.. Train loss: 0.0823.. Train accuracy: 0.9707.. Top-3 train accuracy: 0.9990.. Test loss: 0.3735.. Test accuracy: 0.9055.. Top-3 test accuracy: 0.9860\n",
      "Epoch 42/100.. Time per epoch: 113.0997.. Average time per step: 0.2893.. Train loss: 0.0783.. Train accuracy: 0.9720.. Top-3 train accuracy: 0.9989.. Test loss: 0.3921.. Test accuracy: 0.9029.. Top-3 test accuracy: 0.9860\n",
      "Epoch 43/100.. Time per epoch: 112.8680.. Average time per step: 0.2887.. Train loss: 0.0745.. Train accuracy: 0.9738.. Top-3 train accuracy: 0.9988.. Test loss: 0.3612.. Test accuracy: 0.9089.. Top-3 test accuracy: 0.9871\n",
      "Epoch 44/100.. Time per epoch: 112.7620.. Average time per step: 0.2884.. Train loss: 0.0726.. Train accuracy: 0.9735.. Top-3 train accuracy: 0.9992.. Test loss: 0.3727.. Test accuracy: 0.9060.. Top-3 test accuracy: 0.9886\n",
      "Epoch 45/100.. Time per epoch: 112.9574.. Average time per step: 0.2889.. Train loss: 0.0738.. Train accuracy: 0.9736.. Top-3 train accuracy: 0.9991.. Test loss: 0.3846.. Test accuracy: 0.9032.. Top-3 test accuracy: 0.9860\n",
      "Epoch 46/100.. Time per epoch: 113.0474.. Average time per step: 0.2891.. Train loss: 0.0690.. Train accuracy: 0.9760.. Top-3 train accuracy: 0.9991.. Test loss: 0.4020.. Test accuracy: 0.9008.. Top-3 test accuracy: 0.9860\n",
      "Epoch 47/100.. Time per epoch: 113.1228.. Average time per step: 0.2893.. Train loss: 0.0684.. Train accuracy: 0.9767.. Top-3 train accuracy: 0.9991.. Test loss: 0.3875.. Test accuracy: 0.9048.. Top-3 test accuracy: 0.9875\n",
      "Epoch 48/100.. Time per epoch: 112.8884.. Average time per step: 0.2887.. Train loss: 0.0644.. Train accuracy: 0.9773.. Top-3 train accuracy: 0.9991.. Test loss: 0.3903.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9854\n",
      "Epoch 49/100.. Time per epoch: 112.8441.. Average time per step: 0.2886.. Train loss: 0.0645.. Train accuracy: 0.9772.. Top-3 train accuracy: 0.9991.. Test loss: 0.3977.. Test accuracy: 0.9060.. Top-3 test accuracy: 0.9873\n",
      "Epoch 50/100.. Time per epoch: 112.9351.. Average time per step: 0.2888.. Train loss: 0.0646.. Train accuracy: 0.9772.. Top-3 train accuracy: 0.9992.. Test loss: 0.3897.. Test accuracy: 0.9071.. Top-3 test accuracy: 0.9867\n",
      "Epoch 51/100.. Time per epoch: 112.7946.. Average time per step: 0.2885.. Train loss: 0.0587.. Train accuracy: 0.9795.. Top-3 train accuracy: 0.9994.. Test loss: 0.3996.. Test accuracy: 0.9087.. Top-3 test accuracy: 0.9866\n",
      "Epoch 52/100.. Time per epoch: 112.8441.. Average time per step: 0.2886.. Train loss: 0.0578.. Train accuracy: 0.9800.. Top-3 train accuracy: 0.9992.. Test loss: 0.4119.. Test accuracy: 0.9059.. Top-3 test accuracy: 0.9849\n",
      "Epoch 53/100.. Time per epoch: 112.6090.. Average time per step: 0.2880.. Train loss: 0.0607.. Train accuracy: 0.9788.. Top-3 train accuracy: 0.9994.. Test loss: 0.4081.. Test accuracy: 0.9053.. Top-3 test accuracy: 0.9869\n",
      "Epoch 54/100.. Time per epoch: 112.6020.. Average time per step: 0.2880.. Train loss: 0.0547.. Train accuracy: 0.9809.. Top-3 train accuracy: 0.9995.. Test loss: 0.3942.. Test accuracy: 0.9069.. Top-3 test accuracy: 0.9856\n",
      "Epoch 55/100.. Time per epoch: 112.9037.. Average time per step: 0.2888.. Train loss: 0.0533.. Train accuracy: 0.9812.. Top-3 train accuracy: 0.9996.. Test loss: 0.3946.. Test accuracy: 0.9091.. Top-3 test accuracy: 0.9874\n",
      "Epoch 56/100.. Time per epoch: 112.6162.. Average time per step: 0.2880.. Train loss: 0.0514.. Train accuracy: 0.9824.. Top-3 train accuracy: 0.9995.. Test loss: 0.4330.. Test accuracy: 0.9029.. Top-3 test accuracy: 0.9849\n",
      "Epoch 57/100.. Time per epoch: 112.9411.. Average time per step: 0.2889.. Train loss: 0.0571.. Train accuracy: 0.9794.. Top-3 train accuracy: 0.9994.. Test loss: 0.4360.. Test accuracy: 0.9001.. Top-3 test accuracy: 0.9848\n",
      "Epoch 58/100.. Time per epoch: 112.9766.. Average time per step: 0.2889.. Train loss: 0.0539.. Train accuracy: 0.9812.. Top-3 train accuracy: 0.9996.. Test loss: 0.4140.. Test accuracy: 0.9046.. Top-3 test accuracy: 0.9870\n",
      "Epoch 59/100.. Time per epoch: 112.8613.. Average time per step: 0.2886.. Train loss: 0.0463.. Train accuracy: 0.9836.. Top-3 train accuracy: 0.9996.. Test loss: 0.4382.. Test accuracy: 0.8993.. Top-3 test accuracy: 0.9860\n",
      "Epoch 60/100.. Time per epoch: 112.9702.. Average time per step: 0.2889.. Train loss: 0.0519.. Train accuracy: 0.9819.. Top-3 train accuracy: 0.9996.. Test loss: 0.4069.. Test accuracy: 0.9094.. Top-3 test accuracy: 0.9878\n",
      "Epoch 61/100.. Time per epoch: 112.7905.. Average time per step: 0.2885.. Train loss: 0.0487.. Train accuracy: 0.9824.. Top-3 train accuracy: 0.9996.. Test loss: 0.4005.. Test accuracy: 0.9093.. Top-3 test accuracy: 0.9869\n",
      "Epoch 62/100.. Time per epoch: 112.5288.. Average time per step: 0.2878.. Train loss: 0.0479.. Train accuracy: 0.9831.. Top-3 train accuracy: 0.9995.. Test loss: 0.4116.. Test accuracy: 0.9053.. Top-3 test accuracy: 0.9875\n",
      "Epoch 63/100.. Time per epoch: 112.5636.. Average time per step: 0.2879.. Train loss: 0.0465.. Train accuracy: 0.9835.. Top-3 train accuracy: 0.9996.. Test loss: 0.3912.. Test accuracy: 0.9137.. Top-3 test accuracy: 0.9863\n",
      "Epoch 64/100.. Time per epoch: 112.6140.. Average time per step: 0.2880.. Train loss: 0.0445.. Train accuracy: 0.9847.. Top-3 train accuracy: 0.9997.. Test loss: 0.3980.. Test accuracy: 0.9099.. Top-3 test accuracy: 0.9887\n",
      "Epoch 65/100.. Time per epoch: 112.9100.. Average time per step: 0.2888.. Train loss: 0.0433.. Train accuracy: 0.9850.. Top-3 train accuracy: 0.9994.. Test loss: 0.4116.. Test accuracy: 0.9077.. Top-3 test accuracy: 0.9866\n",
      "Epoch 66/100.. Time per epoch: 112.5846.. Average time per step: 0.2879.. Train loss: 0.0429.. Train accuracy: 0.9848.. Top-3 train accuracy: 0.9996.. Test loss: 0.4094.. Test accuracy: 0.9122.. Top-3 test accuracy: 0.9870\n",
      "Epoch 67/100.. Time per epoch: 112.8351.. Average time per step: 0.2886.. Train loss: 0.0401.. Train accuracy: 0.9858.. Top-3 train accuracy: 0.9997.. Test loss: 0.4139.. Test accuracy: 0.9082.. Top-3 test accuracy: 0.9868\n",
      "Epoch 68/100.. Time per epoch: 112.6685.. Average time per step: 0.2882.. Train loss: 0.0376.. Train accuracy: 0.9865.. Top-3 train accuracy: 0.9996.. Test loss: 0.4711.. Test accuracy: 0.9023.. Top-3 test accuracy: 0.9854\n",
      "Epoch 69/100.. Time per epoch: 112.5682.. Average time per step: 0.2879.. Train loss: 0.0496.. Train accuracy: 0.9823.. Top-3 train accuracy: 0.9995.. Test loss: 0.4321.. Test accuracy: 0.9065.. Top-3 test accuracy: 0.9850\n",
      "Epoch 70/100.. Time per epoch: 112.9659.. Average time per step: 0.2889.. Train loss: 0.0377.. Train accuracy: 0.9870.. Top-3 train accuracy: 0.9997.. Test loss: 0.4331.. Test accuracy: 0.9074.. Top-3 test accuracy: 0.9859\n",
      "Epoch 71/100.. Time per epoch: 112.8890.. Average time per step: 0.2887.. Train loss: 0.0371.. Train accuracy: 0.9866.. Top-3 train accuracy: 0.9997.. Test loss: 0.4286.. Test accuracy: 0.9092.. Top-3 test accuracy: 0.9876\n",
      "Epoch 72/100.. Time per epoch: 112.7029.. Average time per step: 0.2882.. Train loss: 0.0429.. Train accuracy: 0.9852.. Top-3 train accuracy: 0.9998.. Test loss: 0.3997.. Test accuracy: 0.9141.. Top-3 test accuracy: 0.9884\n",
      "Epoch 73/100.. Time per epoch: 113.0615.. Average time per step: 0.2892.. Train loss: 0.0360.. Train accuracy: 0.9871.. Top-3 train accuracy: 0.9997.. Test loss: 0.4191.. Test accuracy: 0.9139.. Top-3 test accuracy: 0.9869\n",
      "Epoch 74/100.. Time per epoch: 113.1344.. Average time per step: 0.2893.. Train loss: 0.0404.. Train accuracy: 0.9857.. Top-3 train accuracy: 0.9997.. Test loss: 0.4428.. Test accuracy: 0.9061.. Top-3 test accuracy: 0.9876\n",
      "Epoch 75/100.. Time per epoch: 112.7803.. Average time per step: 0.2884.. Train loss: 0.0380.. Train accuracy: 0.9872.. Top-3 train accuracy: 0.9997.. Test loss: 0.4330.. Test accuracy: 0.9075.. Top-3 test accuracy: 0.9859\n",
      "Epoch 76/100.. Time per epoch: 112.8555.. Average time per step: 0.2886.. Train loss: 0.0360.. Train accuracy: 0.9876.. Top-3 train accuracy: 0.9997.. Test loss: 0.4500.. Test accuracy: 0.9086.. Top-3 test accuracy: 0.9868\n",
      "Epoch 77/100.. Time per epoch: 112.8129.. Average time per step: 0.2885.. Train loss: 0.0334.. Train accuracy: 0.9880.. Top-3 train accuracy: 0.9998.. Test loss: 0.4477.. Test accuracy: 0.9050.. Top-3 test accuracy: 0.9887\n",
      "Epoch 78/100.. Time per epoch: 112.9629.. Average time per step: 0.2889.. Train loss: 0.0377.. Train accuracy: 0.9864.. Top-3 train accuracy: 0.9998.. Test loss: 0.4661.. Test accuracy: 0.9046.. Top-3 test accuracy: 0.9872\n",
      "Epoch 79/100.. Time per epoch: 112.8381.. Average time per step: 0.2886.. Train loss: 0.0377.. Train accuracy: 0.9867.. Top-3 train accuracy: 0.9997.. Test loss: 0.4304.. Test accuracy: 0.9073.. Top-3 test accuracy: 0.9867\n",
      "Epoch 80/100.. Time per epoch: 113.0107.. Average time per step: 0.2890.. Train loss: 0.0331.. Train accuracy: 0.9886.. Top-3 train accuracy: 0.9998.. Test loss: 0.4552.. Test accuracy: 0.9045.. Top-3 test accuracy: 0.9867\n",
      "Epoch 81/100.. Time per epoch: 112.8068.. Average time per step: 0.2885.. Train loss: 0.0348.. Train accuracy: 0.9880.. Top-3 train accuracy: 0.9998.. Test loss: 0.4551.. Test accuracy: 0.9062.. Top-3 test accuracy: 0.9863\n",
      "Epoch 82/100.. Time per epoch: 112.5329.. Average time per step: 0.2878.. Train loss: 0.0339.. Train accuracy: 0.9883.. Top-3 train accuracy: 0.9997.. Test loss: 0.4382.. Test accuracy: 0.9079.. Top-3 test accuracy: 0.9867\n",
      "Epoch 83/100.. Time per epoch: 112.8313.. Average time per step: 0.2886.. Train loss: 0.0324.. Train accuracy: 0.9891.. Top-3 train accuracy: 0.9998.. Test loss: 0.4335.. Test accuracy: 0.9107.. Top-3 test accuracy: 0.9865\n",
      "Epoch 84/100.. Time per epoch: 112.6410.. Average time per step: 0.2881.. Train loss: 0.0283.. Train accuracy: 0.9899.. Top-3 train accuracy: 0.9999.. Test loss: 0.4540.. Test accuracy: 0.9087.. Top-3 test accuracy: 0.9866\n",
      "Epoch 85/100.. Time per epoch: 112.5976.. Average time per step: 0.2880.. Train loss: 0.0386.. Train accuracy: 0.9866.. Top-3 train accuracy: 0.9997.. Test loss: 0.4448.. Test accuracy: 0.9068.. Top-3 test accuracy: 0.9859\n",
      "Epoch 86/100.. Time per epoch: 112.8488.. Average time per step: 0.2886.. Train loss: 0.0343.. Train accuracy: 0.9879.. Top-3 train accuracy: 0.9998.. Test loss: 0.4673.. Test accuracy: 0.9060.. Top-3 test accuracy: 0.9879\n",
      "Epoch 87/100.. Time per epoch: 112.8758.. Average time per step: 0.2887.. Train loss: 0.0332.. Train accuracy: 0.9890.. Top-3 train accuracy: 0.9998.. Test loss: 0.4233.. Test accuracy: 0.9122.. Top-3 test accuracy: 0.9872\n",
      "Epoch 88/100.. Time per epoch: 112.9115.. Average time per step: 0.2888.. Train loss: 0.0258.. Train accuracy: 0.9907.. Top-3 train accuracy: 0.9999.. Test loss: 0.4351.. Test accuracy: 0.9110.. Top-3 test accuracy: 0.9864\n",
      "Epoch 89/100.. Time per epoch: 112.7238.. Average time per step: 0.2883.. Train loss: 0.0295.. Train accuracy: 0.9897.. Top-3 train accuracy: 0.9999.. Test loss: 0.4612.. Test accuracy: 0.9058.. Top-3 test accuracy: 0.9876\n",
      "Epoch 90/100.. Time per epoch: 112.3289.. Average time per step: 0.2873.. Train loss: 0.0309.. Train accuracy: 0.9895.. Top-3 train accuracy: 0.9998.. Test loss: 0.4773.. Test accuracy: 0.9055.. Top-3 test accuracy: 0.9864\n",
      "Epoch 91/100.. Time per epoch: 112.3752.. Average time per step: 0.2874.. Train loss: 0.0286.. Train accuracy: 0.9902.. Top-3 train accuracy: 0.9998.. Test loss: 0.4389.. Test accuracy: 0.9084.. Top-3 test accuracy: 0.9874\n",
      "Epoch 92/100.. Time per epoch: 112.2581.. Average time per step: 0.2871.. Train loss: 0.0334.. Train accuracy: 0.9886.. Top-3 train accuracy: 0.9997.. Test loss: 0.4770.. Test accuracy: 0.9042.. Top-3 test accuracy: 0.9856\n",
      "Epoch 93/100.. Time per epoch: 112.3953.. Average time per step: 0.2875.. Train loss: 0.0309.. Train accuracy: 0.9891.. Top-3 train accuracy: 0.9999.. Test loss: 0.4516.. Test accuracy: 0.9074.. Top-3 test accuracy: 0.9876\n",
      "Epoch 94/100.. Time per epoch: 111.9781.. Average time per step: 0.2864.. Train loss: 0.0330.. Train accuracy: 0.9887.. Top-3 train accuracy: 0.9998.. Test loss: 0.4567.. Test accuracy: 0.9073.. Top-3 test accuracy: 0.9857\n",
      "Epoch 95/100.. Time per epoch: 111.9020.. Average time per step: 0.2862.. Train loss: 0.0261.. Train accuracy: 0.9906.. Top-3 train accuracy: 0.9999.. Test loss: 0.4158.. Test accuracy: 0.9142.. Top-3 test accuracy: 0.9872\n",
      "Epoch 96/100.. Time per epoch: 112.0188.. Average time per step: 0.2865.. Train loss: 0.0242.. Train accuracy: 0.9914.. Top-3 train accuracy: 0.9998.. Test loss: 0.4676.. Test accuracy: 0.9062.. Top-3 test accuracy: 0.9852\n",
      "Epoch 97/100.. Time per epoch: 111.9803.. Average time per step: 0.2864.. Train loss: 0.0262.. Train accuracy: 0.9909.. Top-3 train accuracy: 0.9997.. Test loss: 0.4368.. Test accuracy: 0.9126.. Top-3 test accuracy: 0.9876\n",
      "Epoch 98/100.. Time per epoch: 111.8205.. Average time per step: 0.2860.. Train loss: 0.0266.. Train accuracy: 0.9907.. Top-3 train accuracy: 0.9999.. Test loss: 0.4828.. Test accuracy: 0.9085.. Top-3 test accuracy: 0.9866\n",
      "Epoch 99/100.. Time per epoch: 111.7124.. Average time per step: 0.2857.. Train loss: 0.0289.. Train accuracy: 0.9898.. Top-3 train accuracy: 0.9998.. Test loss: 0.4859.. Test accuracy: 0.9047.. Top-3 test accuracy: 0.9860\n",
      "Epoch 100/100.. Time per epoch: 111.4761.. Average time per step: 0.2851.. Train loss: 0.0294.. Train accuracy: 0.9903.. Top-3 train accuracy: 0.9999.. Test loss: 0.4609.. Test accuracy: 0.9127.. Top-3 test accuracy: 0.9866\n"
     ]
    }
   ],
   "source": [
    "#train the model\n",
    "model.to(device)\n",
    "\n",
    "steps = 0\n",
    "running_loss = 0\n",
    "for epoch in range(epochs):\n",
    "    \n",
    "    since = time.time()\n",
    "    \n",
    "    train_accuracy = 0\n",
    "    top3_train_accuracy = 0 \n",
    "    for inputs, labels in trainloader:\n",
    "        steps += 1\n",
    "        # Move input and label tensors to the default device\n",
    "        inputs, labels = inputs.to(device), labels.to(device)\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        logps = model.forward(inputs)\n",
    "        loss = criterion(logps, labels)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        running_loss += loss.item()\n",
    "        \n",
    "        # calculate train top-1 accuracy\n",
    "        ps = torch.exp(logps)\n",
    "        top_p, top_class = ps.topk(1, dim=1)\n",
    "        equals = top_class == labels.view(*top_class.shape)\n",
    "        train_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "        \n",
    "        # Calculate train top-3 accuracy\n",
    "        np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "        target_numpy = labels.cpu().numpy()\n",
    "        top3_train_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "        \n",
    "    time_elapsed = time.time() - since\n",
    "    \n",
    "    test_loss = 0\n",
    "    test_accuracy = 0\n",
    "    top3_test_accuracy = 0\n",
    "    model.eval()\n",
    "    with torch.no_grad():\n",
    "        for inputs, labels in testloader:\n",
    "            inputs, labels = inputs.to(device), labels.to(device)\n",
    "            logps = model.forward(inputs)\n",
    "            batch_loss = criterion(logps, labels)\n",
    "\n",
    "            test_loss += batch_loss.item()\n",
    "\n",
    "            # Calculate test top-1 accuracy\n",
    "            ps = torch.exp(logps)\n",
    "            top_p, top_class = ps.topk(1, dim=1)\n",
    "            equals = top_class == labels.view(*top_class.shape)\n",
    "            test_accuracy += torch.mean(equals.type(torch.FloatTensor)).item()\n",
    "            \n",
    "            # Calculate test top-3 accuracy\n",
    "            np_top3_class = ps.topk(3, dim=1)[1].cpu().numpy()\n",
    "            target_numpy = labels.cpu().numpy()\n",
    "            top3_test_accuracy += np.mean([1 if target_numpy[i] in np_top3_class[i] else 0 for i in range(0, len(target_numpy))])\n",
    "\n",
    "    print(f\"Epoch {epoch+1}/{epochs}.. \"\n",
    "          f\"Time per epoch: {time_elapsed:.4f}.. \"\n",
    "          f\"Average time per step: {time_elapsed/len(trainloader):.4f}.. \"\n",
    "          f\"Train loss: {running_loss/len(trainloader):.4f}.. \"\n",
    "          f\"Train accuracy: {train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Top-3 train accuracy: {top3_train_accuracy/len(trainloader):.4f}.. \"\n",
    "          f\"Test loss: {test_loss/len(testloader):.4f}.. \"\n",
    "          f\"Test accuracy: {test_accuracy/len(testloader):.4f}.. \"\n",
    "          f\"Top-3 test accuracy: {top3_test_accuracy/len(testloader):.4f}\")\n",
    "\n",
    "    train_stats = train_stats.append({'Epoch': epoch, 'Time per epoch':time_elapsed, 'Avg time per step': time_elapsed/len(trainloader), 'Train loss' : running_loss/len(trainloader), 'Train accuracy': train_accuracy/len(trainloader), 'Train top-3 accuracy':top3_train_accuracy/len(trainloader),'Test loss' : test_loss/len(testloader), 'Test accuracy': test_accuracy/len(testloader), 'Test top-3 accuracy':top3_test_accuracy/len(testloader)}, ignore_index=True)\n",
    "\n",
    "    running_loss = 0\n",
    "    model.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_stats.to_csv('train_log_DenseNet121_Mish.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
